package org.martin.snsrs.crawler;

import java.io.UnsupportedEncodingException;
import java.util.LinkedList;
import java.util.UUID;
import java.util.regex.Pattern;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class myCrawler extends WebCrawler{
	
	Pattern fileType;
	
	Pattern eventsPattern;
	
	Pattern eventPattern;
	
	Pattern guangzhouPattern;
	
	Pattern locations;
	
	HTMLInfoExtractor extractor;
	
	MyList uuids;
	
	int pageNum;
	
	public myCrawler(){
		fileType = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" 
	            + "|png|tiff?|mid|mp2|mp3|mp4"
	            + "|wav|avi|mov|mpeg|ram|m4v|pdf" 
	            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
		eventsPattern = Pattern.compile(".*www.douban.com/event/.*");
		guangzhouPattern = Pattern.compile("http://guangzhou.douban.com/.*");
		locations = Pattern.compile("http://www.douban.com/location/.*");
		eventPattern = Pattern.compile("http://www.douban.com/event/[0-9]*/?");
		extractor = new HTMLInfoExtractor();
		uuids = MyList.getInstance();
	}
	
	@Override
	public boolean shouldVisit(WebURL webURL){
		boolean isVisit = false;
		String url = webURL.getURL();
		if(!fileType.matcher(url).matches()){
			
			if(eventsPattern.matcher(url).matches()
					|| guangzhouPattern.matcher(url).matches()
					|| locations.matcher(url).matches()){
				if( !uuids.contains(url)){
					isVisit = true;
					uuids.add(url);
				}
				else{
					System.out.println("already access this page");
				}
			}
		}
		return isVisit;
	}
	
	public void visit(Page page){
		if(page.getParseData() instanceof HtmlParseData){
			try {
				String string = new String(page.getContentData(), page.getContentCharset());
				extractor.setCleanerString(string, page.getWebURL().getURL().toLowerCase());
				extractor.extract();
			} catch (UnsupportedEncodingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}
	
}
